# Thomas J. Leeper
# Aarhus University
# May 22, 2014

# Need to Evaluate (NE) AND DK RESPONSES

#setwd("C:/Users/thomas/Downloads/NE and Opinionation/")
#setwd("C:/Users/Thomas/Documents/Dropbox/Research/Opinion Stability/NE and Opinionation")

library("foreign")
library("car")
library("xtable")
    options("xtable.booktabs"=TRUE)
    options("xtable.caption.placement"='top')
    options("xtable.comment"=FALSE)
    options("xtable.floating"=FALSE)
    options("xtable.sanitize.text.function"=identity)

library("devtools")
source_url("https://raw.github.com/leeper/simpleout/master/R/coefpaste.r")
source_url("https://raw.github.com/leeper/mcode/master/R/mergeNA.r")
source_url("https://raw.github.com/leeper/mcode/master/R/mcode.r")

texstars <- function(a,b) ifelse(abs(a/b)>qnorm(0.975), '*', '\\hspace{1ex}')
olsout <- function(obj){
    c(coefpaste(obj$coef[,1],obj$coef[,2], stars_val=texstars),
      n=length(obj$residuals),
      `$\\sigma$`=sprintf('%1.2f',obj$sigma),
      `Adjusted $R^2$`=sprintf('%1.2f',obj$adj.r.squared))
}


###############################
### ANES 2008-2009 ANALYSIS ###
###############################

data <- read.dta("data/anes2008-2009panel.dta")

# DEMOGRAPHICS/COVARIATES
dem <- data.frame(matrix(nrow=dim(data)[1]))
dem$female <- recode(data$der01, "'1. Male'=0;'2. Female'=1;else=NA")
dem$age <- data$der02
dem$hisp <- recode(data$der03, "'1. Hispanic'=1;'0. Non-Hispanic'=0;else=NA")
dem$white <- recode(data$der03a, "'1. white'=1;'0. not white'=0;else=NA")
dem$black <- recode(data$der03b, "'1. African American'=1;'0. not African American'=0;else=NA")
dem$region <- recode(data$wgtregion, "'1. Northeast'=1;'2. Midwest'=2;'3. South'=3;'4. West'=4;else=NA",as.factor.result=TRUE)
dem$married <- recode(data$cpq12, "	'1. Married'=1;
									'2. Widowed'=0;
									'3. Divorced'=0;
									'4. Separated'=0;
									'5.Never married'=0;
									else=NA")
dem$divsep <- recode(data$cpq12, "	'1. Married'=0;
									'2. Widowed'=0;
									'3. Divorced'=1;
									'4. Separated'=1;
									'5.Never married'=0;
									else=NA")
dem$widow <- recode(data$cpq12, "	'1. Married'=0;
									'2. Widowed'=1;
									'3. Divorced'=0;
									'4. Separated'=0;
									'5.Never married'=0;
									else=NA")
dem$hhsize <- recode(data$rhhsize, "-6=NA")-1 # HH size greater than 1
dem$edu <- recode(data$der05, '	"1. No high school diploma"=0;
								"2. High school diploma"=.25;
								"3. Some college, no bachelor\'s degree"=.5;
								"4. Bachelor\'s degree"=.75;
								"5. Graduate degree"=1;
								else=NA', as.factor.result=FALSE) # this is different from the educ variable used in other ANES time-series
dem$HH_nonenglish <- recode(data$cpq14a, "'1. Yes'=1;'2. No'=0; else=NA")
dem$student <- recode(data$cpq16, "'1. A full-time college student'=1; '2. A part-time college student'; '3. Not a college student'=0; else=NA")
dem$working <- recode(data$cpq17, "	'1. Working - as a paid employee'=1;
									'2. Working - self-employed'=1;
									'3. Not working - on temporary layoff from a job'=0;
									'4. Not working - looking for work'=0;
									'5. Not working - retired'=0;
									'6. Not working - disabled'=0;
									'7. Not working - other'=0;
									else=NA")
dem$retired <- recode(data$cpq17, "	'1. Working - as a paid employee'=0;
									'2. Working - self-employed'=0;
									'3. Not working - on temporary layoff from a job'=0;
									'4. Not working - looking for work'=0;
									'5. Not working - retired'=1;
									'6. Not working - disabled'=0;
									'7. Not working - other'=0;
									else=NA")
dem$ownrent <- recode(data$der07, "'1. R owns home'=1;'2. R rents home'=0;'3. Other arrangement'=0;else=NA")

partyid1 <- recode(data$der08w1, "	'0. Strong Democrat'=1;
										'1. Not very strong Democrat'=.67;
										'2. Independent Democrat'=.33;
										'3. Independent-Independent'=0;
										'4. Independent Republican'=-.33;
										'5. Not very strong Republican'=-.67;
										'6. Strong Republican'=-1;
										else=NA", as.factor.result=FALSE)
partyid9 <- recode(data$der08w9, "	'0. Strong Democrat'=1;
										'1. Not very strong Democrat'=.67;
										'2. Independent Democrat'=.33;
										'3. Independent-Independent'=0;
										'4. Independent Republican'=-.33;
										'5. Not very strong Republican'=-.67;
										'6. Strong Republican'=-1;
										else=NA", as.factor.result=FALSE)
partyid10 <- recode(data$der08w10, "'0. Strong Democrat'=1;
										'1. Not very strong Democrat'=.67;
										'2. Independent Democrat'=.33;
										'3. Independent-Independent'=0;
										'4. Independent Republican'=-.33;
										'5. Not very strong Republican'=-.67;
										'6. Strong Republican'=-1;
										else=NA", as.factor.result=FALSE)
partyid11 <- recode(data$der08w11, "'0. Strong Democrat'=1;
										'1. Not very strong Democrat'=.67;
										'2. Independent Democrat'=.33;
										'3. Independent-Independent'=0;
										'4. Independent Republican'=-.33;
										'5. Not very strong Republican'=-.67;
										'6. Strong Republican'=-1;
										else=NA", as.factor.result=FALSE)
dem$partyid <- partyid11 # least missing data
partyid17 <- recode(data$der08w17, "'0. Strong Democrat'=1;
										'1. Not very strong Democrat'=.67;
										'2. Independent Democrat'=.33;
										'3. Independent-Independent'=0;
										'4. Independent Republican'=-.33;
										'5. Not very strong Republican'=-.67;
										'6. Strong Republican'=-1;
										else=NA", as.factor.result=FALSE)
partyid19 <- recode(data$der08w19, "'0. Strong Democrat'=1;
										'1. Not very strong Democrat'=.67;
										'2. Independent Democrat'=.33;
										'3. Independent-Independent'=0;
										'4. Independent Republican'=-.33;
										'5. Not very strong Republican'=-.67;
										'6. Strong Republican'=-1;
										else=NA", as.factor.result=FALSE)
ideology1 <- recode(data$der09w1, "	'1. Extremely Liberal'=1;
										'2. Liberal'=.67;
										'3. Slightly Liberal'=.33;
										'4. Moderate'=0;
										'5. Slightly Conservative'=-.33;
										'6. Conservative'=-.67;
										'7. Extremely Conservative'=-1;
										else=NA", as.factor.result=FALSE)
ideology2 <- recode(data$der09w2, "	'1. Extremely Liberal'=1;
										'2. Liberal'=.67;
										'3. Slightly Liberal'=.33;
										'4. Moderate'=0;
										'5. Slightly Conservative'=-.33;
										'6. Conservative'=-.67;
										'7. Extremely Conservative'=-1;
										else=NA", as.factor.result=FALSE)
ideology6 <- recode(data$der09w6, "	'1. Extremely Liberal'=1;
										'2. Liberal'=.67;
										'3. Slightly Liberal'=.33;
										'4. Moderate'=0;
										'5. Slightly Conservative'=-.33;
										'6. Conservative'=-.67;
										'7. Extremely Conservative'=-1;
										else=NA", as.factor.result=FALSE)
ideology10 <- recode(data$der09w10, "'1. Extremely Liberal'=1;
										'2. Liberal'=.67;
										'3. Slightly Liberal'=.33;
										'4. Moderate'=0;
										'5. Slightly Conservative'=-.33;
										'6. Conservative'=-.67;
										'7. Extremely Conservative'=-1;
										else=NA", as.factor.result=FALSE)
ideology11 <- recode(data$der09w11, "'1. Extremely Liberal'=1;
										'2. Liberal'=.67;
										'3. Slightly Liberal'=.33;
										'4. Moderate'=0;
										'5. Slightly Conservative'=-.33;
										'6. Conservative'=-.67;
										'7. Extremely Conservative'=-1;
										else=NA", as.factor.result=FALSE)
dem$ideology <- ideology11 # least missing data
dem$rel_catholic <- recode(data$der22, "'1. Protestant'=0; '2. Catholic'=1; '3. Jewish'=0; '4. Other'=0; '5. No Religion'=0; else=NA")
dem$rel_protestant <- recode(data$der22, "'1. Protestant'=1; '2. Catholic'=0; '3. Jewish'=0; '4. Other'=0; '5. No Religion'=0; else=NA")
dem$rel_jewish <- recode(data$der22, "'1. Protestant'=0; '2. Catholic'=0; '3. Jewish'=1; '4. Other'=0; '5. No Religion'=0; else=NA")
dem$rel_norelig <- recode(data$der22, "'1. Protestant'=0; '2. Catholic'=0; '3. Jewish'=0; '4. Other'=0; '5. No Religion'=1; else=NA")
dem$interest <- recode(data$rqpol, "'1. Not interested at all'=0;
									'2. Slightly interested'=.25;
									'3. Moderately interested'=.5;
									'4. Very interested'=.75;
									'5. Extremely interested'=1;
									else=NA", as.factor.result=FALSE)

# need to evaluate (NE) measures
table(data$w11ze1, useNA="always") # 3 NAs; 88 terminated interviews
table(data$w11ze2, useNA="always") # 6 NAs; 88 terminated interviews

ne1 <- recode(data$w11ze1,"'-9. Refused'=NA;
			'-7. No answer'=NA; 
			'-6. Not asked, unit non-response'=NA; 
			'-5. Not asked, terminated'=NA; 
			'-4. Error, see documentation'=NA;
			'-3. Restricted access'=NA;
			'-2. Missing, see documentation'=NA;
			'-1. Inapplicable'=NA;
			'1. Almost everything'=1;
			'2. Many things'=.67;
			'3. Some things'=.33;
			'4. Very few things'=0", as.factor.result=FALSE)
ne2stem <- recode(data$w11ze2,"'-9. Refused'=NA;
			'-7. No answer'=NA; 
			'-6. Not asked, unit non-response'=NA; 
			'-5. Not asked, terminated'=NA; 
			'-4. Error, see documentation'=NA;
			'-3. Restricted access'=NA;
			'-2. Missing, see documentation'=NA;
			'-1. Inapplicable'=NA;
			'1. Fewer opinions'=-1;
			'2. About the same number of opinions'=0;
			'3. More opinions'=1", as.factor.result=FALSE)
ne2a <- recode(data$w11ze3a,"'-9. Refused'=NA;
			'-7. No answer'=NA; 
			'-6. Not asked, unit non-response'=NA; 
			'-5. Not asked, terminated'=NA; 
			'-4. Error, see documentation'=NA;
			'-3. Restricted access'=NA;
			'-2. Missing, see documentation'=NA;
			'-1. Inapplicable'=NA;
			'1. A lot'=1;
			'2. Somewhat'=0", as.factor.result=FALSE) # more
ne2b <- recode(data$w11ze3b,"'-9. Refused'=NA;
			'-7. No answer'=NA; 
			'-6. Not asked, unit non-response'=NA; 
			'-5. Not asked, terminated'=NA; 
			'-4. Error, see documentation'=NA;
			'-3. Restricted access'=NA;
			'-2. Missing, see documentation'=NA;
			'-1. Inapplicable'=NA;
			'1. A lot'=1;
			'2. Somewhat'=0", as.factor.result=FALSE) # fewer
ne2leaf <- mergeNA(ne2a,ne2b)
ne2 <- mcode(ne2stem,ne2leaf,recodes="c(0,NA)=.5; c(1,1)=1; c(1,0)=.75; c(-1,1)=0; c(-1,0)=.25")
dem$ne <- rowMeans(cbind(ne1,ne2),na.rm=TRUE)

# need for cognition (nc) variables
nc1 <- recode(data$w11ze6,"'-9. Refused'=NA;
			'-7. No answer'=NA; 
			'-6. Not asked, unit non-response'=NA; 
			'-5. Not asked, terminated'=NA; 
			'-4. Error, see documentation'=NA;
			'-3. Restricted access'=NA;
			'-2. Missing, see documentation'=NA;
			'-1. Inapplicable'=NA;
			'1. Simple'=0;
			'2. Complex'=1", as.factor.result=FALSE)
nc2stem <- recode(data$w11ze4,"'-9. Refused'=NA;
			'-7. No answer'=NA; 
			'-6. Not asked, unit non-response'=NA; 
			'-5. Not asked, terminated'=NA; 
			'-4. Error, see documentation'=NA;
			'-3. Restricted access'=NA;
			'-2. Missing, see documentation'=NA;
			'-1. Inapplicable'=NA;
			'1. Like'=1;
			'2. Dislike'=-1;
			'3. Neither like nor dislike'=0", as.factor.result=FALSE)
nc2a <- recode(data$w11ze5a,"'-9. Refused'=NA;
			'-7. No answer'=NA; 
			'-6. Not asked, unit non-response'=NA; 
			'-5. Not asked, terminated'=NA; 
			'-4. Error, see documentation'=NA;
			'-3. Restricted access'=NA;
			'-2. Missing, see documentation'=NA;
			'-1. Inapplicable'=NA;
			'1. A lot'=1;
			'2. Just somewhat'=0", as.factor.result=FALSE) # more
nc2b <- recode(data$w11ze5b,"'-9. Refused'=NA;
			'-7. No answer'=NA; 
			'-6. Not asked, unit non-response'=NA; 
			'-5. Not asked, terminated'=NA; 
			'-4. Error, see documentation'=NA;
			'-3. Restricted access'=NA;
			'-2. Missing, see documentation'=NA;
			'-1. Inapplicable'=NA;
			'1. A lot'=1;
			'2. Just somewhat'=0", as.factor.result=FALSE) # less
nc2leaf <- mergeNA(nc2a,nc2b)
nc2 <- mcode(nc2stem,nc2leaf,recodes="c(0,NA)=.5; c(1,1)=1; c(1,0)=.75; c(-1,1)=0; c(-1,0)=.25")
dem$nc <- rowMeans(cbind(nc1,nc2),na.rm=TRUE)

# CLEAN UP
dem[,1] <- NULL
nonmissing <- !is.na(dem$ne)
dem <- dem[nonmissing,]

# political knowledge
know1 <- recode(data$w11wv1, "'1. Arizona, correct'=1; else=0") # McCain's home state
know2 <- recode(data$w11wv2, "'1. Illinois, correct'=1; else=0") # Obama's home state
know3 <- recode(data$w11wv3, "'1. Christian, correct'=1; else=0") # Obama's religion; may want to check sensitivity
know4 <- recode(data$w11wv4, "'1. Christian, correct'=1; else=0") # McCain's religion
know5 <- recode(data$w11wv5, "'1. A state legislature, correct'=1;else=0") # know Obama's previous job
know6 <- recode(data$w11wv6, "'2. The U.S. military, correct'=1;else=0") # know McCain's previous job
know7 <- recode(data$w11wv7, "2=1;else=0") # know presidential term limit
know8 <- recode(data$w11wv8, "6=1;else=0") # know U.S. senator term length
know9 <- recode(data$w11wv9, "2=1;else=0") # know number of Senators/state
know10 <- recode(data$w11wv10, "2=1;else=0") # know MoC term length
know11 <- recode(data$w11wv11, "'3. Speaker of the House of Representatives, correct'=1;else=0") # know presidential succession order
know12 <- recode(data$w11wv12, "'2. Two-thirds, correct'=1;else=0") # know veto override percentage
know <- data.frame(cbind(know1,know3,know3,know4,know4,know5,know6,know7,know8,know9,know10,know11,know12))[nonmissing,]
know$knowtotal <- rowSums(know, na.rm=TRUE)
#know_obamarelig <- data.frame(cbind(know1,know3,know4,know4,know5,know6,know7,know8,know9,know10,know11,know12))[nonmissing,]
#know_tmp <- rowSums(know_obamarelig, na.rm=TRUE)
#cor(know$knowtotal, know_tmp) # 0.99 correlation
dem <- cbind(dem,know)
rm(know)


# OPINION QUESTIONS
if(TRUE){
.recodes_NA <- "'-9. Refused'=1;
			'-7. No answer'=1; 
			'-6. Not asked, unit non-response'=NA; 
			'-5. Not asked, terminated'=NA; 
			'-4. Error, see documentation'=NA;
			'-3. Restricted access'=NA;
			'-2. Missing, see documentation'=NA;
			'-1. Inapplicable'=NA;
			'1. Favor'=0;
			'2. Oppose'=0;
			'3. Neither favor nor oppose'=1"
} else { # same effect for just giving middle-category responses, but no effect of NE on just No Answer/Refused (small n)
.recodes_NA <- "'-9. Refused'=0;
				'-7. No answer'=0;
				'-6. Not asked, unit non-response'=NA; 
				'-5. Not asked, terminated'=NA; 
				'-4. Error, see documentation'=NA;
				'-3. Restricted access'=NA;
				'-2. Missing, see documentation'=NA;
				'-1. Inapplicable'=NA;
				'1. Favor'=0;
				'2. Oppose'=0;
				'3. Neither favor nor oppose'=1"
}

# support ban on gay marriage
ban1NA <- recode(data$w1p1,.recodes_NA, as.factor.result=FALSE)
ban2NA <- recode(data$w10p1,.recodes_NA, as.factor.result=FALSE)

# raise high-income taxes
tax1NA <- recode(data$w1p4,.recodes_NA, as.factor.result=FALSE)
tax2NA <- recode(data$w10p4,.recodes_NA, as.factor.result=FALSE)

# senior drug benefit
drug1NA <- recode(data$w1p10,.recodes_NA, as.factor.result=FALSE)
drug2NA <- recode(data$w10p10,.recodes_NA, as.factor.result=FALSE)

# gov't payment of healthcare
hc1NA <- recode(data$w1p13,.recodes_NA, as.factor.result=FALSE)
hc2NA <- recode(data$w10p13,.recodes_NA, as.factor.result=FALSE)

# favor suspend habeas for terror suspects
terror1NA <- recode(data$w1p16,.recodes_NA, as.factor.result=FALSE)
terror2NA <- recode(data$w10p16,.recodes_NA, as.factor.result=FALSE)

# support court order for wiretap
wiretap1NA <- recode(data$w1p19,.recodes_NA, as.factor.result=FALSE)
wiretap2NA <- recode(data$w10p19,.recodes_NA, as.factor.result=FALSE)

# support guest worker program
worker1NA <- recode(data$w1p22,.recodes_NA, as.factor.result=FALSE)
worker2NA <- recode(data$w10p22,.recodes_NA, as.factor.result=FALSE)

# support path to citizenship
citizen1NA <- recode(data$w1p25,.recodes_NA, as.factor.result=FALSE)
citizen2NA <- recode(data$w10p25,.recodes_NA, as.factor.result=FALSE)


newdata <- as.data.frame(cbind(	ban1NA,ban2NA,tax1NA,tax2NA,drug1NA,drug2NA,
								hc1NA,hc2NA,terror1NA,terror2NA,wiretap1NA,wiretap2NA,
								worker1NA,worker2NA,citizen1NA,citizen2NA))[nonmissing,]

# need descriptives for original opinion questions (percents in DK / no answer / middle category)
anes0809desctab <- cbind(
	coefpaste(	apply(data[,c("w1p1","w1p4","w1p10","w1p13","w1p16","w1p19","w1p22","w1p25")],2,function(x)
					table(x)["-7. No answer"]),
				apply(data[,c("w1p1","w1p4","w1p10","w1p13","w1p16","w1p19","w1p22","w1p25")],2,function(x)
					prop.table(table(x))["-7. No answer"]*100), c(0,2)),
	coefpaste(	apply(data[,c("w1p1","w1p4","w1p10","w1p13","w1p16","w1p19","w1p22","w1p25")],2,function(x)
					table(x)["3. Neither favor nor oppose"]),
				apply(data[,c("w1p1","w1p4","w1p10","w1p13","w1p16","w1p19","w1p22","w1p25")],2,function(x)
					prop.table(table(x))["3. Neither favor nor oppose"]*100), c(0,2)),
	coefpaste(	apply(data[,c("w10p1","w10p4","w10p10","w10p13","w10p16","w10p19","w10p22","w10p25")],2,function(x)
					table(x)["-7. No answer"]),
				apply(data[,c("w10p1","w10p4","w10p10","w10p13","w10p16","w10p19","w10p22","w10p25")],2,function(x)
					prop.table(table(x))["-7. No answer"]*100), c(0,2)),
	coefpaste(	apply(data[,c("w10p1","w10p4","w10p10","w10p13","w10p16","w10p19","w10p22","w10p25")],2,function(x)
					table(x)["3. Neither favor nor oppose"]),
				apply(data[,c("w10p1","w10p4","w10p10","w10p13","w10p16","w10p19","w10p22","w10p25")],2,function(x)
					prop.table(table(x))["3. Neither favor nor oppose"]*100), c(0,2)) )
anes0809desctab <- rbind(rep(c("No Answer","Neither favor nor oppose"),2),anes0809desctab)
rownames(anes0809desctab) <- c("Response","Gay Marriage Ban","High Income Tax Increase","Senior Drug Benefit","Gov't Health Insurance",
						"Suspend Habeas","Court Order for Wiretap","Guest Worker Program","Path to Citizenship")
colnames(anes0809desctab) <- c("T1"," ","T2"," ")
print(xtable(anes0809desctab, caption="Response Distributions, by Question", align=c("l",rep("p{1in}",4)), label='apptab:anes0809dist'),
    file='tables/anes0809dist.txt')

###########################
### NE/Missing Analysis ###
###########################

with(dem, coefpaste(mean(ne,na.rm=TRUE),sd(ne,na.rm=TRUE)))
round(median(dem$ne,na.rm=TRUE),2)

nulltab <- 
rbind(	c(with(newdata, round(prop.table(table(ban1NA)),3))[2], with(newdata, round(prop.table(table(ban2NA)),3))[2]),
		c(with(newdata, round(prop.table(table(tax1NA)),3))[2], with(newdata, round(prop.table(table(tax2NA)),3))[2]),
		c(with(newdata, round(prop.table(table(drug1NA)),3))[2], with(newdata, round(prop.table(table(drug2NA)),3))[2]),
		c(with(newdata, round(prop.table(table(hc1NA)),3))[2], with(newdata, round(prop.table(table(hc2NA)),3))[2]),
		c(with(newdata, round(prop.table(table(terror1NA)),3))[2], with(newdata, round(prop.table(table(terror2NA)),3))[2]),
		c(with(newdata, round(prop.table(table(wiretap1NA)),3))[2], with(newdata, round(prop.table(table(wiretap2NA)),3))[2]),
		c(with(newdata, round(prop.table(table(worker1NA)),3))[2], with(newdata, round(prop.table(table(worker2NA)),3))[2]),
		c(with(newdata, round(prop.table(table(citizen1NA)),3))[2], with(newdata, round(prop.table(table(citizen2NA)),3))[2]) ) * 100
colnames(nulltab) <- c("T1","T2")
rownames(nulltab) <- c(	"Gay Marriage Ban","High Income Tax Increase","Senior Drug Benefit","Gov't Health Insurance",
						"Suspend Habeas for Terror Suspects","Court Order for Wiretap","Guest Worker Program","Path to Citizenship")
# print percent of NA/DK/middle category responses for each issue
cat('\\caption{Percent Nonattitudes on ANES 2008-2009 Panel Questions}\\label{tab:props2008-2009}', file='tables/props2008-2009.txt')
print(xtable(nulltab, align=c('l','r','r'), digits=1), file='tables/props2008-2009.txt', append=TRUE)

#NA1temp <- with(newdata, ban1NA + tax1NA + drug1NA + hc1NA + terror1NA + wiretap1NA + worker1NA + citizen1NA)
NA1temp <- rowSums(newdata[,c("ban1NA","tax1NA","drug1NA","hc1NA","terror1NA","wiretap1NA","worker1NA","citizen1NA")], na.rm=TRUE)
#NA2temp <- with(newdata, ban2NA + tax2NA + drug2NA + hc2NA + terror2NA + wiretap2NA + worker2NA + citizen2NA)
NA2temp <- rowSums(newdata[,c("ban2NA","tax2NA","drug2NA","hc2NA","terror2NA","wiretap2NA","worker2NA","citizen2NA")], na.rm=TRUE)
NAtotal <- rowSums(newdata, na.rm=TRUE)/16 # number of NAs among everyone (thus, lots of zeroes)
NAs <- (NA1temp + NA2temp)/16 # number of NAs among those who offered at least one NA (thus, no zeroes)
NA1 <- NA1temp/8
NA2 <- NA2temp/8

# SIMPLE MODELS OF TOTAL NONRESPONSE
ne <- dem$ne
nc <- dem$nc
simp1 <- lm(NA1 ~ ne,data=newdata)
simp2 <- lm(NA2 ~ ne,data=newdata)
simp3 <- lm(NAs ~ ne,data=newdata)
simp4 <- lm(NAtotal ~ ne,data=newdata)

predict(simp1, newdata=data.frame(ne=seq(0,1,length.out=10)))
predict(simp2, newdata=data.frame(ne=seq(0,1,length.out=10)))
predict(simp3, newdata=data.frame(ne=seq(0,1,length.out=10)))
predict(simp4, newdata=data.frame(ne=seq(0,1,length.out=10)))

simp11 <- lm(NA1 ~ ne + nc,data=newdata)
simp12 <- lm(NA2 ~ ne + nc,data=newdata)
simp13 <- lm(NAs ~ ne + nc,data=newdata)
simp14 <- lm(NAtotal ~ ne + nc,data=newdata)

pdata <- expand.grid(ne=seq(0,1,length.out=10), nc=seq(0,1,length.out=10))
matrix(predict(simp11, newdata=pdata),nrow=10) # rows=NE, cols=NC
matrix(predict(simp12, newdata=pdata),nrow=10)
matrix(predict(simp13, newdata=pdata),nrow=10)
matrix(predict(simp14, newdata=pdata),nrow=10)



# RICHER MODELS OF TOTAL NONRESPONSES
# per reviewer comments, fold partyid and ideology to be 0=independent/moderate and 1=strong/extreme
formNA1 <- NA1 ~ ne + nc + female + age + white + black + hisp + edu + 
					married + divsep + widow + student + working + retired + 
					I(abs(partyid)) + I(abs(ideology)) + interest + knowtotal
formNA2 <- NA2 ~ ne + nc + female + age + white + black + hisp + edu + 
					married + divsep + widow + student + working + retired + 
					I(abs(partyid)) + I(abs(ideology)) + interest + knowtotal
formNAs <- NAs ~ ne + nc + female + age + white + black + hisp + edu + 
					married + divsep + widow + student + working + retired + 
					I(abs(partyid)) + I(abs(ideology)) + interest + knowtotal
formNAtotal <- NAtotal ~ ne + nc + female + age + white + black + hisp + edu + 
					married + divsep + widow + student + working + retired + 
					I(abs(partyid)) + I(abs(ideology)) + interest + knowtotal

full1 <- summary(lm(formNA1,data=dem))
full2 <- summary(lm(formNA2,data=dem))
full3 <- #summary(lm(formNAs,data=dem))
		summary(lm(formNAtotal,data=dem))
fulltab <- cbind(olsout(full1),
                 olsout(full2),
                 olsout(full3))
colnames(fulltab) <- c("Nonattitudes (T1)","Nonattitudes (T2)","Nonattitudes (Combined)")
rownames(fulltab) <- c("Intercept","NE","NC","Female","Age","White","Black","Hispanic","Education",
						"Married","Divorced","Widowed","Student","Working","Retired","PartyID","Ideology","Interest","Knowledge",
						"n","$\\sigma$","Adjusted $R^2$")
cat('\\caption{Non-informative Responding on Opinion Questions, ANES 2008--2009}\\label{tab:full2008-2009}', file='tables/full2008-2009.txt')
print(xtable(fulltab, align=c("l",rep("r",3))), file='tables/full2008-2009.txt', append=TRUE)


# ROBUSTNESS CHECK: TREAT AS PANEL DATA
if(FALSE){
    dem$respondent <- rownames(dem)
    panel <- dem
    for(i in 1:15)
        panel <- rbind(panel,dem)
    panel$item <- sort(rep(seq(1,16),dim(dem)[1]))
    panel$itemNA <- with(newdata, c(ban1NA, ban2NA, tax1NA, tax2NA, drug1NA, drug2NA, hc1NA, hc2NA, terror1NA, terror2NA, wiretap1NA, wiretap2NA, worker1NA, worker2NA, citizen1NA, citizen2NA))
    #panel <- plm.data(panel, indexes=c("respondent","item"))

    panelformula1 <- itemNA ~ ne + nc + female + age + white + black + hisp + edu + 
                        married + divsep + widow + student + working + retired + 
                        partyid + ideology + interest + knowtotal + factor(item)
    #summary(lm(panelformula1, data=panel))
}



### EXTREMITY ANALYSIS ###
# This can be rerun with the middle category ('Neither favor nor oppose') set to NA
.recodes_ex1 <- "'-9. Refused'=NA;
			'-7. No answer'=NA; 
			'-6. Not asked, unit non-response'=NA; 
			'-5. Not asked, terminated'=NA; 
			'-4. Error, see documentation'=NA;
			'-3. Restricted access'=NA;
			'-2. Missing, see documentation'=NA;
			'-1. Inapplicable'=NA;
			'1. Favor'=1;
			'2. Oppose'=1;
			'3. Neither favor nor oppose'=NA"
.recodes_favor <- "'-9. Refused'=1;
			'-7. No answer'=1; 
			'-6. Not asked, unit non-response'=1; 
			'-5. Not asked, terminated'=1; 
			'-4. Error, see documentation'=1;
			'-3. Restricted access'=1;
			'-2. Missing, see documentation'=1;
			'-1. Inapplicable'=1;
			'1. A great deal'=1;
			'2. Moderately'=.5;
			'3. A little'=0"
.recodes_oppose <- "'-9. Refused'=1;
			'-7. No answer'=1; 
			'-6. Not asked, unit non-response'=1; 
			'-5. Not asked, terminated'=1; 
			'-4. Error, see documentation'=1;
			'-3. Restricted access'=1;
			'-2. Missing, see documentation'=1;
			'-1. Inapplicable'=1;
			'1. A great deal'=1;
			'2. Moderately'=.5;
			'3. A little'=0"

# support ban on gay marriage
ban1ex <- 	recode(data$w1p1,.recodes_ex1, as.factor.result=FALSE) *
			recode(data$w1p_f_2,.recodes_favor, as.factor.result=FALSE) *
			recode(data$w1p_o_2,.recodes_oppose, as.factor.result=FALSE)
ban2ex <- 	recode(data$w10p1,.recodes_ex1, as.factor.result=FALSE) *
			recode(data$w10p2_favor,.recodes_favor, as.factor.result=FALSE) *
			recode(data$w10p2_oppose,.recodes_oppose, as.factor.result=FALSE)

# raise high-income taxes
tax1ex <- 	recode(data$w1p4,.recodes_ex1, as.factor.result=FALSE) *
			recode(data$w1p_f_5,.recodes_favor, as.factor.result=FALSE) *
			recode(data$w1p_o_5,.recodes_oppose, as.factor.result=FALSE)
tax2ex <- 	recode(data$w10p4,.recodes_ex1, as.factor.result=FALSE) *
			recode(data$w10p5_favor,.recodes_favor, as.factor.result=FALSE) *
			recode(data$w10p5_oppose,.recodes_oppose, as.factor.result=FALSE)

# senior drug benefit
drug1ex <- 	recode(data$w1p10,.recodes_ex1, as.factor.result=FALSE) *
			recode(data$w1p_f_11,.recodes_favor, as.factor.result=FALSE) *
			recode(data$w1p_o_11,.recodes_oppose, as.factor.result=FALSE)
drug2ex <- 	recode(data$w10p10,.recodes_ex1, as.factor.result=FALSE) *
			recode(data$w10p11_favor,.recodes_favor, as.factor.result=FALSE) *
			recode(data$w10p11_oppose,.recodes_oppose, as.factor.result=FALSE)

# gov't payment of healthcare
hc1ex <- 	recode(data$w1p13,.recodes_ex1, as.factor.result=FALSE) *
			recode(data$w1p_f_14,.recodes_favor, as.factor.result=FALSE) *
			recode(data$w1p_o_14,.recodes_oppose, as.factor.result=FALSE)
hc2ex <- recode(data$w10p13,.recodes_ex1, as.factor.result=FALSE)

# favor suspend habeas for terror suspects
terror1ex <- 	recode(data$w1p16,.recodes_ex1, as.factor.result=FALSE) *
				recode(data$w1p_f_17,.recodes_favor, as.factor.result=FALSE) *
				recode(data$w1p_o_17,.recodes_oppose, as.factor.result=FALSE)
terror2ex <-recode(data$w10p16,.recodes_ex1, as.factor.result=FALSE) *
			recode(data$w10p17_favor,.recodes_favor, as.factor.result=FALSE) *
			recode(data$w10p17_oppose,.recodes_oppose, as.factor.result=FALSE)

# support court order for wiretap
wiretap1ex <- 	recode(data$w1p19,.recodes_ex1, as.factor.result=FALSE) *
				recode(data$w1p_f_20,.recodes_favor, as.factor.result=FALSE) *
				recode(data$w1p_o_20,.recodes_oppose, as.factor.result=FALSE)
wiretap2ex <- 	recode(data$w10p19,.recodes_ex1, as.factor.result=FALSE) *
				recode(data$w10p20_favor,.recodes_favor, as.factor.result=FALSE) *
				recode(data$w10p20_oppose,.recodes_oppose, as.factor.result=FALSE)

# support guest worker program
worker1ex <- 	recode(data$w1p22,.recodes_ex1, as.factor.result=FALSE) *
				recode(data$w1p_f_23,.recodes_favor, as.factor.result=FALSE) *
				recode(data$w1p_o_23,.recodes_oppose, as.factor.result=FALSE)
worker2ex <- 	recode(data$w10p22,.recodes_ex1, as.factor.result=FALSE) *
				recode(data$w10p23_favor,.recodes_favor, as.factor.result=FALSE) *
				recode(data$w10p23_oppose,.recodes_oppose, as.factor.result=FALSE)

# support path to citizenship
citizen1ex <- 	recode(data$w1p25,.recodes_ex1, as.factor.result=FALSE) *
				recode(data$w1p_f_26,.recodes_favor, as.factor.result=FALSE) *
				recode(data$w1p_o_26,.recodes_oppose, as.factor.result=FALSE)
citizen2ex <- 	recode(data$w10p25,.recodes_ex1, as.factor.result=FALSE) *
				recode(data$w10p26_favor,.recodes_favor, as.factor.result=FALSE) *
				recode(data$w10p26_oppose,.recodes_oppose, as.factor.result=FALSE)


exdata <- as.data.frame(cbind(	ban1ex,ban2ex,tax1ex,tax2ex,drug1ex,drug2ex,
								hc1ex,hc2ex,terror1ex,terror2ex,wiretap1ex,wiretap2ex,
								worker1ex,worker2ex,citizen1ex,citizen2ex))[nonmissing,]

exdata$exAvg1 <- rowMeans(exdata[,grepl("1ex",names(exdata))],na.rm=TRUE)
exdata$exAvg2 <- rowMeans(exdata[,grepl("2ex",names(exdata))],na.rm=TRUE)
exdata$exAvg <- rowMeans(exdata,na.rm=TRUE)
#table(exdata$exAvg,dem$ne)

formban1ex <- exdata$exAvg1 ~ ne + nc + female + age + white + black + hisp + edu + 
					married + divsep + widow + student + working + retired + 
					I(abs(partyid)) + I(abs(ideology)) + interest + knowtotal
formban2ex <- exdata$exAvg2 ~ ne + nc + female + age + white + black + hisp + edu + 
					married + divsep + widow + student + working + retired + 
					I(abs(partyid)) + I(abs(ideology)) + interest + knowtotal
formbanex <- exdata$exAvg ~ ne + nc + female + age + white + black + hisp + edu + 
					married + divsep + widow + student + working + retired + 
					I(abs(partyid)) + I(abs(ideology)) + interest + knowtotal
lmex0809_1 <- summary(lm(formban1ex,data=dem))
lmex0809_2 <- summary(lm(formban2ex,data=dem))
lmex0809 <- summary(lm(formbanex,data=dem))

extab <- cbind(	olsout(lmex0809_1),
				olsout(lmex0809_2),
                olsout(lmex0809))
colnames(extab) <- c("Extremity (T1)","Extremity (T2)","Extremity (Combined)")
rownames(extab) <- c(	"Intercept","NE","NC","Female","Age","White","Black","Hispanic","Education",
						"Married","Divorced","Widowed","Student","Working","Retired","PartyID","Ideology","Political Interest","Knowledge",
						"n","$\\sigma$","Adjusted $R^2$")

cat('\\caption{Attitude Extremity, ANES 2008-2009}\\label{tab:extremity0809}', file='tables/extremity0809.txt')
print(xtable(extab, align=c("l",rep("r",3))), file='tables/extremity0809.txt', append=TRUE)


    
    
    
    
    
    
#################################################################################
### TEST WHETHER EFFECTS HOLD ACROSS OTHER DATA (ANES 1998, 2000, 2004, 2008) ###
#################################################################################

d1998 <- read.dta("data/anes1998pilot.dta", convert.factors=FALSE)
d2000 <- read.dta("data/anes2000.dta", convert.factors=FALSE)
d2004 <- read.dta("data/anes2004.dta", convert.factors=FALSE)
d2008 <- read.dta("data/anes2008.dta", convert.factors=FALSE)

# 1998 Pilot
table(d1998$V98p387, useNA="always") # 0 NAs on NE1
table(d1998$V98p388, useNA="always") # 11 NAs on NE2

ne1 <- recode(d1998$V98p387, "1=1;2=.67;3=.33;4=0;8:9=NA;0=NA") # NAs/DKs to NA; 0=Not asked
ne2stem <- recode(d1998$V98p388, "1=-1;3=0;5=1;8:9=NA;0=NA") # NAs/DKs to NA; 0=Not asked
ne2leaf1 <- recode(d1998$V98p389, "1=1;2=0;8:9=NA;0=NA") # NAs/DKs to NA; 0=Not asked
ne2leaf2 <- recode(d1998$V98p390, "1=1;2=0;8:9=NA;0=NA") # NAs/DKs to NA; 0=Not asked
ne2leaf <- mergeNA(ne2leaf1,ne2leaf2)
ne2 <- mcode(ne2stem,ne2leaf,recodes="c(0,NA)=.5; c(1,1)=1; c(1,0)=.75; c(-1,1)=0; c(-1,0)=.25")
ne <- rowMeans(cbind(ne1,ne2),na.rm=TRUE)

nc1 <- recode(d1998$V98p395, "1=0;5=1;8:9=NA;") # simple or complex problems
nc2stem <- recode(d1998$V98p392, "1=1;3=-1;5=0;8:9=NA")
nc2leaf1 <- recode(d1998$V98p393, "1=1;5=0;c(0,8,9)=NA")
nc2leaf2 <- recode(d1998$V98p394, "1=1;5=0;c(0,8,9)=NA")
nc2leaf <- mergeNA(nc2leaf1,nc2leaf2)
nc2 <- mcode(nc2stem,nc2leaf,recodes="c(0,NA)=.5; c(1,1)=1; c(1,0)=.75; c(-1,1)=0; c(-1,0)=.25")
nc <- rowMeans(cbind(nc1,nc2),na.rm=TRUE)

interest <- recode(d1998$V98p376, "1=1;2=.67;3=.33;4=0;8=0;9=0;else=NA")
female <- recode(d1998$V98p031, "2=1;1=0")
age <- d1998$V98p030
white <- recode(d1998$V98p431, "86=1;else=0")
black <- recode(d1998$V98p431, "87=1;else=0")
hispanic <- recode(d1998$V98p434, "1=1;else=0")
education <- recode(d1998$V98p424, "98=NA") # years of schooling; refused = NA; 17 is 17 or more years
married <- recode(d1998$V98p423, "1=1;else=0")
divorced <- recode(d1998$V98p423, "3=1;4=1;else=0") # divorced/separated
widowed <- recode(d1998$V98p423, "2=1;else=0")
tmp1 <- recode(d1998$V98p427, "8=1;else=0") # first mention
tmp2 <- recode(d1998$V98p428, "8=1;else=0") # second mention
student <- as.numeric(apply(cbind(tmp1,tmp2),1,any)) # OR-logical if student
rm(tmp1,tmp2)
tmp1 <- recode(d1998$V98p427, "1=1;else=0") # first mention
tmp2 <- recode(d1998$V98p428, "1=1;else=0") # second mention
working <- as.numeric(apply(cbind(tmp1,tmp2),1,any)) # OR-logical if working
rm(tmp1,tmp2)
tmp1 <- recode(d1998$V98p427, "5=1;else=0") # first mention
tmp2 <- recode(d1998$V98p428, "5=1;else=0") # second mention
retired <- as.numeric(apply(cbind(tmp1,tmp2),1,any)) # OR-logical if retired
rm(tmp1,tmp2)
partyid <- recode(d1998$V98p329, "0=1;1=.67;2=.33;3=0;4=-.33;5=-.67;6=-1;else=0")
ideology <- recode(d1998$V98p296b, "1=1;3=0;5=-1;else=0") # combined summary of two formats (three-category)
# knowledge # no interviewer rating

know1NAa <- recode(d1998$V98p344, "8=1;9=1;0=NA;else=0") # know Supreme Court
know1NAb <- recode(d1998$V98p348, "8=1;9=1;0=NA;else=0") # know Supreme Court
know1NA <- mergeNA(know1NAa,know1NAb)

know2NAa <- recode(d1998$V98p345, "8=1;9=1;0=NA;else=0") # know judicial nominations
know2NAb <- recode(d1998$V98p350, "8=1;9=1;0=NA;else=0") # know Supreme Court
know2NA <- mergeNA(know2NAa,know2NAb)

know3NAa <- recode(d1998$V98p346, "8=1;9=1;0=NA;else=0") # party control House
know3NAb <- recode(d1998$V98p352, "8=1;9=1;0=NA;else=0") # know Supreme Court
know3NA <- mergeNA(know3NAa,know3NAb)

know4NAa <- recode(d1998$V98p347, "8=1;9=1;0=NA;else=0") # party control Senate
know4NAb <- recode(d1998$V98p354, "8=1;9=1;0=NA;else=0") # know Supreme Court
know4NA <- mergeNA(know4NAa,know4NAb)
rm(know1NAa, know2NAa, know3NAa, know4NAa)
rm(know1NAb, know2NAb, know3NAb, know4NAb)
knowNAscale <- rowSums(sapply(ls(pattern='know[[:digit:]]+NA'), get))/4
rm(know1NA, know2NA, know3NA, know4NA)




#Spending/services: 98P332 (standard wording/responses); 98P331 ("haven't thought about it much" option)
spend_1 <- recode(d1998$V98p331, "1:7=0;0=1;8=1;else=NA")
spend_1b <- recode(d1998$V98p331, "0=1;1:9=0") # "haven't thought about it much"
spend_2 <- recode(d1998$V98p332, "1:7=0;8=1;else=NA") # initial DK
spend <- mergeNA(spend_1,spend_2)

# extremity analysis
spend_1 <- recode(d1998$V98p331, "1=1;2=.67;3=.33;4=0;5=.33;6=.67;7=1;else=NA")
spend_2 <- recode(d1998$V98p332, "1=1;2=.67;3=.33;4=0;5=.33;6=.67;7=1;else=NA") # initial DK
ex_spend <- mergeNA(spend_1,spend_2)


data1998 <- data.frame(cbind(	spend, ex_spend,
								ne, nc, interest, female, age, white, black, hispanic, education,
								married, divorced, widowed, student, working, retired,
								partyid, ideology, knowNAscale))
data1998$year <- rep(1998,dim(data1998)[1])



# 2000 (PROVIDES A MIDDLE CATEGORY RESPONSE EXPERIMENT)
table(d2000$V000862, useNA="always") # 4 NAs on NE1 (+5 R's w/o interview)
table(d2000$V000866, useNA="always") # 31 NAs on NE2 (+5 R's w/o interview)
#NE: V000862	V000866
ne1 <- recode(d2000$V000862, "1=1;2=.67;3=.33;4=0;c(0,8,9)=NA")
ne2 <- recode(d2000$V000866, "1=0;2=.25;3=.5;4=.75;5=1;c(0,8,9)=NA")
ne <- rowMeans(cbind(ne1,ne2),na.rm=TRUE)
#NC: V000870	V000871
nc1 <- recode(d2000$V000871,"5=1;1=0;c(0,8,9)=NA")
nc2 <- recode(d2000$V000870,"1=1;2=.75;3=.5;4=.25;5=0;c(0,8,9)=NA")
nc <- rowMeans(cbind(nc1,nc2),na.rm=TRUE)

interest <- recode(d2000$V001367,"1=1;2=.67;3=.33;4=0;8=0;9=0;else=NA")
female <- recode(d2000$V001029, "2=1;1=0")
age <- d2000$V000908
white <- recode(d2000$V001006a, "50=1;else=0") # first mention
black <- recode(d2000$V001006a, "10=1;else=0") # first mention
hispanic <- recode(d2000$V001012, "1=1;else=0")
education <- recode(d2000$V000910, "98:99=NA") # years of schooling; refused/DK/NA = NA; 17 is 17 or more years
married <- recode(d2000$V000909, "1=1;else=0")
divorced <- recode(d2000$V000909, "c(3,4)=1;else=0") # divorced/separated
widowed <- recode(d2000$V000909, "2=1;else=0")
student <- recode(d2000$V000919, "c(18,80,75,81)=1;else=0")
working <- recode(d2000$V000919, "c(10,15,16,17,18,51,61,71,81)=1;else=0")
retired <- recode(d2000$V000919, "c(15,50,51)=1;else=0")
partyid <- recode(d2000$V000523, "0=1;1=.67;2=.33;3=0;4=-.33;5=-.67;6=-1;else=0")
ideology <- recode(d2000$V001370, "1=1;3=0;5=-1;else=0") # three-category
knowledge <- recode(d2000$V001745, "5=0;4=.25;3=.5;2=.75;1=1;else=NA") # interviewer rating

# knowledge DKs
know1NA <- recode(d2000$V001447, "8=1;9=1;else=0") # know Trent Lott
know2NA <- recode(d2000$V001450, "8=1;9=1;else=0") # know William Rehquist
know3NA <- recode(d2000$V001453, "8=1;9=1;else=0") # know Tony Blair
know4NA <- recode(d2000$V001456, "8=1;9=1;else=0") # know Janet Reno
know5NA <- recode(d2000$V001458, "8=1;9=1;else=0") # know Bush state
know6NA <- recode(d2000$V001460, "8=1;9=1;else=0") # know Bush religion
know7NA <- recode(d2000$V001462, "8=1;9=1;else=0") # know Gore state
know8NA <- recode(d2000$V001464, "8=1;9=1;else=0") # know Gore religion
know9NA <- recode(d2000$V001466, "8=1;9=1;else=0") # know Cheney state
know10NA <- recode(d2000$V001468, "8=1;9=1;else=0") # know Cheney religion
know11NA <- recode(d2000$V001470, "8=1;9=1;else=0") # know Lieberman state
know12NA <- recode(d2000$V001472, "8=1;9=1;else=0") # know Lieberman religion
know13NA <- recode(d2000$V001356, "8=1;9=1;else=0") # know House party control
know14NA <- recode(d2000$V001357, "8=1;9=1;else=0") # know House party control
knowNAscale <- rowSums(sapply(ls(pattern='know[[:digit:]]+NA'), get))/14
rm(know1NA, know2NA, know3NA, know4NA, know5NA, know6NA, know7NA, know8NA, know9NA, know10NA, know11NA, know12NA, know13NA, know14NA)


#Spending/services: V000549 (phone); V000545 (in-person)
spend_1 <- recode(d2000$V000545,"1:7=0;8=1;0=1;9=NA")
spend_2a <- recode(d2000$V000549,"1:5=0;7=1;8=1;9=1;0=NA")
spend_2b <- recode(d2000$V000549,"c(1,2,4,5)=0;3=1;7=1;8=1;9=1;0=NA") # include equivocal middle category
spend <- mergeNA(spend_1,spend_2a)
#spend <- mergeNA(spend_1,spend_2b)
#Standard of living: V000619 (phone); V000615 (in-person)
standard_1 <- recode(d2000$V000615,"1:7=0;8=1;0=1;9=NA")
standard_2a <- recode(d2000$V000619,"1:5=0;8=1;0=1;9=NA")
standard_2b <- recode(d2000$V000619,"c(1,2,4,5)=0;3=1;7=1;8=1;9=1;0=NA") # include equivocal middle category
standard <- mergeNA(standard_1,standard_2a)
#standard <- mergeNA(standard_1,standard_2b)
#Aid to blacks: V000644 (phone); V000641 (in-person)
aid_1 <- recode(d2000$V000641,"1:7=0;8=1;0=1;9=NA")
aid_2a <- recode(d2000$V000644,"1:5=0;8=1;0=1;9=NA")
aid_2b <- recode(d2000$V000644,"c(1,2,4,5)=0;3=1;7=1;8=1;9=1;0=NA") # include equivocal middle category
aid <- mergeNA(aid_1,aid_2a)
#aid <- mergeNA(aid_1,aid_2b)

# extremity analysis
tmp1 <- recode(d2000$V000545, "1=1;2=.67;3=.33;4=0;5=.33;6=.67;7=1;else=NA")
tmp2 <- recode(d2000$V000549, "1=1;2=.5;3=0;4=.5;5=1;else=NA")
ex_spend <- mergeNA(tmp1,tmp2)
tmp1 <- recode(d2000$V000615, "1=1;2=.67;3=.33;4=0;5=.33;6=.67;7=1;else=NA")
tmp2 <- recode(d2000$V000619, "1=1;2=.5;3=0;4=.5;5=1;else=NA")
ex_standard <- mergeNA(tmp1,tmp2)
tmp1 <- recode(d2000$V000641, "1=1;2=.67;3=.33;4=0;5=.33;6=.67;7=1;else=NA")
tmp2 <- recode(d2000$V000644, "1=1;2=.5;3=0;4=.5;5=1;else=NA")
ex_aid <- mergeNA(tmp1,tmp2)



data2000 <- data.frame(cbind(	spend, standard, aid, ex_spend, ex_standard, ex_aid,
								ne, nc, interest, female, age, white, black, hispanic, education,
								married, divorced, widowed, student, working, retired,
								partyid, ideology, knowledge, knowNAscale))
data2000$year <- rep(2000,dim(data2000)[1])


# 2004
table(d2004$V045218, useNA="always") # 1 NAs on NE1 (+146 R's w/no post-election)
table(d2004$V045219a, useNA="always") # 0 NAs on NE2 (+157 R's w/no post-election)
#NE: V045218	V045219a
ne1 <- recode(d2004$V045218, "1=1;2=.67;3=.33;4=0;8:9=NA")
ne2 <- recode(d2004$V045219a, "1=0;2=.25;3=.5;4=.75;5=1;c(8,9)=NA")
ne <- rowMeans(cbind(ne1,ne2),na.rm=TRUE)
#NC: V045220a	V045221
nc1 <- recode(d2004$V045221,"5=1;1=0;c(8,9)=NA")
nc2 <- recode(d2004$V045220a,"1=1;2=.75;3=.5;4=.25;5=0;c(8,9)=NA")
nc <- rowMeans(cbind(nc1,nc2),na.rm=TRUE)

interest <- recode(d2004$V045095,"1=1;2=.67;3=.33;4=0;8=0;9=0;else=NA")
female <- recode(d2004$V041109a, "2=1;1=0")
age <- recode(d2004$V043250, "c(98,99,00)=NA")
white <- recode(d2004$V043299a, "50=1;else=0") # first mention
black <- recode(d2004$V043299a, "10=1;else=0") # first mention
tmp1 <- recode(d2004$V043304x, "1:3=1;else=0")
tmp2 <- recode(d2004$V043305, "1=1;else=0")
hispanic <- as.numeric(apply(cbind(tmp1,tmp2),1,any))
rm(tmp1,tmp2)
education <- recode(d2004$V043252,"88:89=NA") # years of schooling; refused/DK/NA = NA; 17 is 17 or more years
married <- recode(d2004$V043251, "1=1;else=0")
divorced <- recode(d2004$V043251, "c(3,4)=1;else=0") # divorced/separated
widowed <- recode(d2004$V043251, "2=1;else=0")
student <- recode(d2004$V043260a, "c(18,75,80,81)=1;else=0")
working <- recode(d2004$V043260a, "c(10,15,16,17,18,51,61,71,81)=1;else=0")
retired <- recode(d2004$V043260a, "c(15,50,51)=1;else=0")
partyid <- recode(d2004$V043116, "0=1;1=.67;2=.33;3=0;4=-.33;5=-.67;6=-1;else=0")
ideology <- recode(d2004$V045118, "1=1;3=0;5=-1;else=0")
knowledge <- recode(d2004$V045303, "5=0;4=.25;3=.5;2=.75;1=1;else=NA") # interviewer rating

know1NA <- recode(d2004$V045162, "8=1;9=1;else=0") # know Dennis Hastert
know2NA <- recode(d2004$V045163, "8=1;9=1;else=0") # know Dick Cheney
know3NA <- recode(d2004$V045164, "8=1;9=1;else=0") # know Tony Blair
know4NA <- recode(d2004$V045165, "8=1;9=1;else=0") # know William Rehnquist
know5NA <- recode(d2004$V045089, "8=1;9=1;else=0") # party control House
know6NA <- recode(d2004$V045090, "8=1;9=1;else=0") # party control Senate
#know7NA <- recode(d2004$V045160a, "8=1;9=1;else=0") # which party more conservative (messy two part question)
knowNAscale <- rowSums(sapply(ls(pattern='know[[:digit:]]+NA'), get))/6
rm(know1NA, know2NA, know3NA, know4NA, know5NA, know6NA)





#Spending/services: V043136
spend <- recode(d2004$V043136, "1:7=0;80=1;88=1;89=1") # 1="haven't thought about this much"; "don't know"; "refused"
#Standard of living: V043152
standard <- recode(d2004$V043152, "1:7=0;80=1;88=1;89=1") # 1="haven't thought about this much"; "don't know"; "refused"
#Aid to blacks: V043158
aid <- recode(d2004$V043158, "1:7=0;80=1;88=1;89=1") # 1="haven't thought about this much"; "don't know"; "refused"

# extremity analysis
ex_spend <- recode(d2004$V043136, "1=1;2=.67;3=.33;4=0;5=.33;6=.67;7=1;else=NA")
ex_standard <- recode(d2004$V043152, "1=1;2=.67;3=.33;4=0;5=.33;6=.67;7=1;else=NA")
ex_aid <- recode(d2004$V043158, "1=1;2=.67;3=.33;4=0;5=.33;6=.67;7=1;else=NA")

data2004 <- data.frame(cbind(	spend, standard, aid, ex_spend, ex_standard, ex_aid,
								ne, nc, interest, female, age, white, black, hispanic, education,
								married, divorced, widowed, student, working, retired,
								partyid, ideology, knowledge, knowNAscale))
data2004$year <- rep(2004,dim(data2004)[1])


# 2008
table(d2008$V085168, useNA="always") # 2 NAs on NE1 (+220 R's w/no post-election)
table(d2008$V085169x, useNA="always") # 17 NAs on NE2 (+220 R's w/no post-election)
#NE: V085168	V085169x
ne1 <- recode(d2008$V085168, "1=1;2=.67;3=.33;4=0;c(-2,-8,-9)=NA")
ne2 <- recode(d2008$V085169x, "1=0;2=.25;3=.5;4=.75;5=1;c(-2,-8,-9)=NA")
ne <- rowMeans(cbind(ne1,ne2),na.rm=TRUE)
#NC: V085170x	V085171 
nc1 <- recode(d2008$V085171,"5=1;1=0;c(-2,-8,-9)=NA")
nc2 <- recode(d2008$V085170x,"1=1;2=.75;3=.5;4=.25;5=0;c(-2,-8,-9)=NA")
nc <- rowMeans(cbind(nc1,nc2),na.rm=TRUE)

interest <- recode(d2008$V085072, "1=1;2=.67;3=.33;4=0;-8=0;-9=0;else=NA")
female <- recode(d2008$V081101, "2=1;1=0")
age <- recode(d2008$V081104, "-8;-9=NA")
white <- recode(d2008$V083251a, "50=1;else=0") # first mention
black <- recode(d2008$V083251a, "10=1;else=0") # first mention
tmp1 <- recode(d2008$V083255, "1:3=1;else=0")
tmp2 <- recode(d2008$V083256, "1=1;else=0")
hispanic <- as.numeric(apply(cbind(tmp1,tmp2),2,any))
rm(tmp1,tmp2)
education <- recode(d2008$V083217,"-8=NA;-9=NA") # years of schooling; refused/DK/NA = NA; 17 is 17 or more years
married <- recode(d2008$V083216x, "1=1;else=0")
divorced <- recode(d2008$V083216x, "c(2,3)=1;else=0") # divorced/separated
widowed <- recode(d2008$V083216x, "4=1;else=0")
student <- recode(d2008$V083222x, "c(18,75,80,81)=1;else=0")
working <- recode(d2008$V083222x, "c(10,15,16,17,18,51,61,71,81)=1;else=0")
retired <- recode(d2008$V083222x, "c(15,50,51)=1;else=0")
partyid <- recode(d2008$V083098x, "0=1;1=.67;2=.33;3=0;4=-.33;5=-.67;6=-1;else=0")
ideology <- recode(d2008$V083069x, "1=1;3=0;5=-1;else=0")
knowledge <- recode(d2008$V085403, "5=0;4=.25;3=.5;2=.75;1=1;else=NA") # interviewer rating

# knowledge scales are still not released
# d2008$V085119a # which party is more conservative (tricky two-part question)
# d2008$V085120 # know Nancy Pelosi
# d2008$V085121 # know Dick Cheney
# d2008$V085122 # know Gordon Brown
# d2008$V085123 # know John Roberts
know1NA <- recode(d2008$V085066, "-8=1;-9=1;else=0") # party conrol House
know2NA <- recode(d2008$V085067, "-8=1;-9=1;else=0") # party conrol Senate
knowNAscale <- rowSums(sapply(ls(pattern='know[[:digit:]]+NA'), get))/2
rm(know1NA, know2NA)



#Spending/services:  V083105 (old) V083108x (new)
spend_1 <- recode(d2008$V083105, "1:7=0;-7=1;-8=1;-9=1;-1=NA") # 1="haven't thought about this much"; "don't know"; "refused"
spend_2 <- recode(d2008$V083108x, "1:7=0;-8=1;-9=1;-1=NA") # 1="don't know"; "refused"
spend <- mergeNA(spend_1,spend_2)
#Standard of living: V083128
standard <- recode(d2008$V083128, "1:7=0;-7=1;-8=1;-9=1;-1=NA") # 1="haven't thought about this much"; "don't know"; "refused"
#Aid to blacks: V083137
aid <- recode(d2008$V083137, "1:7=0;-7=1;-8=1;-9=1") # 1="haven't thought about this much"; "don't know"; "refused"

# extremity analysis
tmp1 <- recode(d2008$V083105, "1=1;2=.67;3=.33;4=0;5=.33;6=.67;7=1;else=NA")
tmp2 <- recode(d2008$V083108x, "1=1;2=.67;3=.33;4=0;5=.33;6=.67;7=1;else=NA")
ex_spend <- mergeNA(tmp1,tmp2)
ex_standard <- recode(d2008$V083128, "1=1;2=.67;3=.33;4=0;5=.33;6=.67;7=1;else=NA")
ex_aid <- recode(d2008$V083137, "1=1;2=.67;3=.33;4=0;5=.33;6=.67;7=1;else=NA")


data2008 <- data.frame(cbind(	spend, standard, aid, ex_spend, ex_standard, ex_aid,
								ne, nc, interest, female, age, white, black, hispanic, education,
								married, divorced, widowed, student, working, retired,
								partyid, ideology, knowledge, knowNAscale))
data2008$year <- rep(2008,dim(data2008)[1])

# merge dataframes
pooled <- merge(data1998, data2000, all=TRUE)
pooled <- merge(pooled, data2004, all=TRUE)
pooled <- merge(pooled, data2008, all=TRUE)

# cleanup
#rm(d1998, d2000, d2004, d2008)
#rm(data1998, data2000, data2004, data2008)
rm(ne1,ne2,ne2stem,ne2leaf1,ne2leaf2,ne2leaf,ne,nc1,nc2,nc2stem,nc2leaf1,nc2leaf2,nc2leaf,nc)
rm(	female,age,white,black,hispanic,education,married,divorced,widowed,
	student,working,retired,partyid,ideology,interest,knowledge)
rm(aid_1,aid_2a,aid_2b,spend_1,spend_1b,spend_2,spend_2a,spend_2b,standard_1,standard_2a,standard_2b)



###############################
## ANES TIME-SERIES ANALYSIS ##
###############################

pooled <- pooled[!pooled$year==1998,] # exclude 1998 (no knowledge variable); results same w/ inclusion

with(pooled, coefpaste(by(ne,year,mean,na.rm=TRUE),by(ne,year,sd,na.rm=TRUE)))

tstab <- cbind(
	sprintf('%1.1f',prop.table(with(pooled, table(year, spend)),1)[,2]*100),
	sprintf('%1.1f',prop.table(with(pooled, table(year, standard)),1)[,2]*100),
	sprintf('%1.1f',prop.table(with(pooled, table(year, aid)),1)[,2]*100) )
tstab <- cbind(tstab,sprintf('%1.1f',rowMeans(matrix(as.numeric(tstab),nrow=nrow(tstab)))))
rownames(tstab) <- c(2000,2004,2008)
colnames(tstab) <- c("Spending/Services","Jobs and Standard of Living","Aid to Blacks","Average")

cat('\\caption{Non-informative Responses as a Percentage of All Responses}\\label{tab:anestsdesc}', file='tables/anestsdesc.txt')
print(xtable(tstab, align=c("l",rep("r",4))), file='tables/anestsdesc.txt', append=TRUE)

# simple models (2000, 2004, 2008)
#summary(lm(spend ~ ne + nc + interest + education + factor(year),data=pooled))
#summary(lm(standard ~ ne + nc + interest + education + factor(year),data=pooled))
#summary(lm(aid ~ ne + nc + interest + education + factor(year),data=pooled))

# full models (2000, 2004, 2008)
tsspend <- 
summary(lm(spend ~	ne + nc + female + age + white + black + hispanic + 
					education + married + divorced + widowed + student +
					working + retired + I(abs(partyid)) + I(abs(ideology)) + interest + knowledge + factor(year), data=pooled))
tsstandard <- 
summary(lm(standard ~ne + nc + female + age + white + black + hispanic + 
					education + married + divorced + widowed + student +
					working + retired + I(abs(partyid)) + I(abs(ideology)) + interest + knowledge + factor(year), data=pooled))
tsaid <- 
summary(lm(aid ~	ne + nc + female + age + white + black + hispanic + 
					education + married + divorced + widowed + student +
					working + retired + I(abs(partyid)) + I(abs(ideology)) + interest + knowledge + factor(year), data=pooled))

tsregtab <- cbind(olsout(tsspend),
                  olsout(tsstandard),
                  olsout(tsaid))
colnames(tsregtab) <- c("Spending/Services","Jobs and Standard of Living","Aid to Blacks")
rownames(tsregtab) <- c("Intercept","NE","NC","Female","Age","White","Black","Hispanic","Education",
							"Married","Divorced","Widowed","Student","Working","Retired","PartyID","Ideology","Political Interest","Knowledge","2004","2008",
							"n","$\\sigma$","Adjusted $R^2$")
cat('\\caption{Non-informative Responding to Opinion Items, ANES 2000, 2004, 2008}\\label{tab:fullts}', file='tables/fullts.txt')
print(xtable(tsregtab, 	align=c("l",rep("r",3))), file='tables/fullts.txt', append=TRUE)

## INTERPRETATION: NE HAS A CONSISTENT, BUT SMALL NEGATIVE EFFECT ON NONATTITUDE EXPRESSION. THOSE HIGHER IN NE ARE MORE LIKELY TO GIVE INFORMATIVE RESPONSES.


##############################################
### ANES TIME SERIES ANALYSIS OF EXTREMITY ###
##############################################

# simple models
#summary(lm(ex_spend ~ ne + nc + interest + education + factor(year),data=pooled))
#summary(lm(ex_standard ~ ne + nc + interest + education + factor(year),data=pooled))
#summary(lm(ex_aid ~ ne + nc + interest + education + factor(year),data=pooled))

ex_tsspend <- 
summary(lm(ex_spend ~	ne + nc + female + age + white + black + hispanic + 
					education + married + divorced + widowed + student +
					working + retired + I(abs(partyid)) + I(abs(ideology)) + interest + knowledge + factor(year), data=pooled))
ex_tsstandard <- 
summary(lm(ex_standard ~ne + nc + female + age + white + black + hispanic + 
					education + married + divorced + widowed + student +
					working + retired + I(abs(partyid)) + I(abs(ideology)) + interest + knowledge + factor(year), data=pooled))
ex_tsaid <- 
summary(lm(ex_aid ~	ne + nc + female + age + white + black + hispanic + 
					education + married + divorced + widowed + student +
					working + retired + I(abs(partyid)) + I(abs(ideology)) + interest + knowledge + factor(year), data=pooled))

                    


ex_tsregtab <- cbind(olsout(ex_tsspend),
                     olsout(ex_tsstandard),
                     olsout(ex_tsaid))
colnames(ex_tsregtab) <- c("Spending/Services","Jobs and Standard of Living","Aid to Blacks")
rownames(ex_tsregtab) <- c("Intercept","NE","NC","Female","Age","White","Black","Hispanic","Education",
							"Married","Divorced","Widowed","Student","Working","Retired","PartyID","Ideology","Political Interest","Knowledge","2004","2008",
							"n","$\\sigma$","Adjusted $R^2$")
cat('\\caption{Attitude Extremity, ANES 2000, 2004, 2008}\\label{tab:extremityts}', file='tables/extremityts.txt')
print(xtable(ex_tsregtab, align=c("l",rep("r",3))), file='tables/extremityts.txt', append=TRUE)

## INTERPRETATION: NE HAS A SMALL BUT POSITIVE EFFECT ON ATTITUDE EXTREMITY (AMONG THOSE GIVING SUBSTANTIVE RESPONSES)


#####################################################################################
### CHECK TO MAKE SURE ne ISN'T JUST PREDICTING ALL DKs USING KNOWLEDGE QUESTIONS ###
#####################################################################################

.recode_know <- '"-9. Refused"=1; "-8. Don\'t Know"=1; "-7. No answer"=1; else=0'
.recode_know2 <- '-9=1; -8=1; -7=1; else=0'
know1NA <- recode(data$w11wv1, .recode_know, as.factor.result=FALSE) # McCain's home state
know2NA <- recode(data$w11wv2, .recode_know, as.factor.result=FALSE) # Obama's home state
know3NA <- recode(data$w11wv3, .recode_know, as.factor.result=FALSE) # Obama's religion
know4NA <- recode(data$w11wv4, .recode_know, as.factor.result=FALSE) # McCain's religion
know5NA <- recode(data$w11wv5, .recode_know, as.factor.result=FALSE) # know Obama's previous job
know6NA <- recode(data$w11wv6, .recode_know, as.factor.result=FALSE) # know McCain's previous job
know7NA <- recode(data$w11wv7, .recode_know2, as.factor.result=FALSE) # know presidential term limit
know8NA <- recode(data$w11wv8, .recode_know2, as.factor.result=FALSE) # know U.S. senator term length
know9NA <- recode(data$w11wv9, .recode_know2, as.factor.result=FALSE) # know number of Senators/state
know10NA <- recode(data$w11wv10, .recode_know2, as.factor.result=FALSE) # know MoC term length
know11NA <- recode(data$w11wv11, .recode_know, as.factor.result=FALSE) # know presidential succession order
know12NA <- recode(data$w11wv12, .recode_know, as.factor.result=FALSE) # know veto override percentage
knowNA <- cbind.data.frame(know1NA,know2NA,know3NA,know4NA,know5NA,know6NA,know7NA,know8NA,know9NA,know10NA,know11NA,know12NA)[nonmissing,]
knowNA$total <- rowSums(knowNA, na.rm=TRUE)/12
rm(know1NA, know2NA, know3NA, know4NA, know5NA, know6NA, know7NA, know8NA, know9NA, know10NA, know11NA, know12NA)

knowNA1998 <- 
summary(lm(knowNAscale ~	ne + nc + female + age + white + black + hispanic + 
					education + married + divorced + widowed + student +
					working + retired + I(abs(partyid)) + I(abs(ideology)) + interest, data=data1998))

tsknowNA2000 <- 
summary(lm(knowNAscale ~	ne + nc + female + age + white + black + hispanic + 
					education + married + divorced + widowed + student +
					working + retired + I(abs(partyid)) + I(abs(ideology)) + interest, data=pooled[pooled$year==2000,]))
tsknowNA2004 <- 
summary(lm(knowNAscale ~	ne + nc + female + age + white + black + hispanic + 
					education + married + divorced + widowed + student +
					working + retired + I(abs(partyid)) + I(abs(ideology)) + interest, data=pooled[pooled$year==2004,]))

tsknowNA2008 <- 
summary(lm(knowNAscale ~	ne + nc + female + age + white + black + hispanic + 
					education + married + divorced + widowed + student +
					working + retired + I(abs(partyid)) + I(abs(ideology)) + interest, data=pooled[pooled$year==2008,]))

knowformNA <- knowNA$total ~ ne + nc + female + age + white + black + hisp + edu + 
					married + divsep + widow + student + working + retired +
					I(abs(partyid)) + I(abs(ideology)) + interest
knowNA0809 <- summary(lm(knowformNA,data=dem))

knowsumtab <- cbind(c(olsout(knowNA1998),
                      DKs=coefpaste(mean(data1998$knowNAscale*100),
                                        sd(data1998$knowNAscale*100),c(1,1))),
                    c(olsout(tsknowNA2000),
                      DKs=coefpaste(mean(pooled$knowNAscale[pooled$year==2000]*100),
                                        sd(pooled$knowNAscale[pooled$year==2000]*100),c(1,1))),
                    c(olsout(tsknowNA2004),
                      DKs=coefpaste(mean(pooled$knowNAscale[pooled$year==2004]*100),
                                        sd(pooled$knowNAscale[pooled$year==2004]*100),c(1,1))),
                    c(olsout(tsknowNA2008)[1:7],'--',olsout(tsknowNA2008)[8:20],
                      DKs=coefpaste(mean(pooled$knowNAscale[pooled$year==2008]*100),
                                        sd(pooled$knowNAscale[pooled$year==2008]*100),c(1,1))),
                    c(olsout(knowNA0809),
                      DKs=coefpaste(mean(knowNA$total*100),
                                        sd(knowNA$total*100),c(1,1))))
colnames(knowsumtab) <- c("1998","2000","2004","2008","2008--2009")
rownames(knowsumtab) <- c("Intercept","NE","NC","Female","Age","White","Black","Hispanic","Education",
							"Married","Divorced","Widowed","Student","Working","Retired","PartyID","Ideology","Political Interest",
							"n","$\\sigma$","Adjusted $R^2$","\\% DKs (SD)")
cat('%\\caption{Nonequivalent Outcome Test of Effects of NE on Knowledge}\\label{tab:knowledgedks}\n', file='tables/knowledgedks.txt')
print(xtable(knowsumtab, align=c('l',rep('r',5))), file='tables/knowledgedks.txt', append=TRUE)








##############################################################
### ANES 1998 Pilot: Supplemental data for online appendix ###
##############################################################

# full model (1998, spending/services only; small sample due to NE only asked of half of respondents)
spend98 <- 
summary(lm(spend ~	ne + nc + female + age + white + black + hispanic + 
					education + married + divorced + widowed + student +
					working + retired + I(abs(partyid)) + I(abs(ideology)) + interest, data=data1998)) # no knowledge variable
ex_spend98 <- 
summary(lm(ex_spend ~	ne + nc + female + age + white + black + hispanic + 
					education + married + divorced + widowed + student +
					working + retired + I(abs(partyid)) + I(abs(ideology)) + interest, data=data1998)) # no knowledge variable

regtab98 <- cbind(olsout(spend98), olsout(ex_spend98))
colnames(regtab98) <- c("Nonattitudes","Attitude Extremity")
rownames(regtab98) <- c("Intercept","NE","NC","Female","Age","White","Black","Hispanic","Education",
							"Married","Divorced","Widowed","Student","Working","Retired","PartyID","Ideology","Political Interest",
							"n","$\\sigma$","Adjusted $R^2$")
cat('\\caption{Nonattitude Expression and Attitude Extremity, ANES 1998 Pilot}\\label{apptab:replication98}', file='tables/replication98.txt')
print(xtable(regtab98, align=c("l",rep("r",2))), file='tables/replication98.txt')




					
##########################################################################
### ANES 1987 PILOT STUDY ANALYSIS OF DKs IN ZALLER/FELDMAN EXPERIMENT ###
##########################################################################

anes1987 <- read.dta("data/ANES 1987 Pilot.dta")

# Form A is post-opinion thought listing
# Form B is stop-and-think

# V872218 # spending/services thought listing
spendserv1 <- recode(anes1987$V872218, "1=0;3=0;5=0;7=1;9=NA;0=NA")
# V872290 # spending/services stop-and-think
spendserv2 <- recode(anes1987$V872290, "1=0;3=0;5=0;8=1;9=NA;0=NA")

# V872269 # jobs/standard of living thought listing
jobs1 <- recode(anes1987$V872269, "1=0;3=0;5=0;7=1;9=NA;0=NA")
# V872292 # jobs/standard of living stop-and-think
jobs2 <- recode(anes1987$V872292, "1=0;3=0;5=0;8=1;9=NA;0=NA")

# V872279	# aid to blacks thought listing
aid1 <- recode(anes1987$V872279, "1=0;3=0;5=0;7=1;8=1;9=NA;0=NA")
# V872301 # aid to blacks stop-and-think
aid2 <- recode(anes1987$V872301, "1=0;5=0;8=1;9=NA;0=NA")


# extremity coding (only three categories, so this might be weird)
exrecodes <- "1=1;3=0;5=1;7:9=NA;0=NA"
ex_spendserv1 <- recode(anes1987$V872218, exrecodes)
ex_spendserv2 <- recode(anes1987$V872290, exrecodes)
ex_jobs1 <- recode(anes1987$V872269, exrecodes)
ex_jobs2 <- recode(anes1987$V872292, exrecodes)
ex_aid1 <- recode(anes1987$V872279, exrecodes)
ex_aid2 <- recode(anes1987$V872301, exrecodes)
rm(exrecodes)

### ANALYSIS OF DKs ###

s1 <- table(spendserv1)
s2 <- table(spendserv2)

j1 <- table(jobs1)
j2 <- table(jobs2)

a1 <- table(aid1)
a2 <- table(aid2)

# descriptives 
sprop1 <- sprintf('%1.1f',(s1[2]/sum(s1))*100)
sprop2 <- sprintf('%1.1f',(s2[2]/sum(s2))*100)
stest <- prop.test(c(s1[2],s2[2]),c(sum(s1),sum(s2)))

jprop1 <- sprintf('%1.1f',(j1[2]/sum(j1))*100)
jprop2 <- sprintf('%1.1f',(j2[2]/sum(j2))*100)
jtest <- prop.test(c(j1[2],j2[2]),c(sum(j1),sum(j2)))

aprop1 <- sprintf('%1.1f',(a1[2]/sum(a1))*100)
aprop2 <- sprintf('%1.1f',(a2[2]/sum(a2))*100)
atest <- prop.test(c(a1[2],a2[2]),c(sum(a1),sum(a2)))

# print
tab <- 
rbind(	c(paste0(sprop1,"\\%"),paste0(sprop2,"\\%"),
            paste0(sprintf('%1.2f',stest$statistic),", p=",sprintf('%1.2f',stest$p.value))),
		c(paste0(jprop1,"\\%"),paste0(jprop2,"\\%"),
            paste0(sprintf('%1.2f',jtest$statistic),", p=",sprintf('%1.2f',jtest$p.value))),
		c(paste0(aprop1,"\\%"),paste0(aprop2,"\\%"),
        paste0(sprintf('%1.2f',atest$statistic),", p=",sprintf('%1.2f',atest$p.value))) )
rownames(tab) <- c("Spending/Services", "Jobs and Standard of Living", "Aid to Blacks")
colnames(tab) <- c("Standard Wording", "Stop-and-Think", "Difference ($\\chi^2$)")

cat('\\caption{DKs Under Standard and Stop-and-Think Wordings, ANES 1989 Pilot}\\label{tab:zallerfeldman}', file='tables/zallerfeldman.txt')
print(xtable(tab, align=c("l",rep("r",3))), file='tables/zallerfeldman.txt', append=TRUE)

### ANALYSIS OF EXTREMITY ###
ex_s1 <- table(ex_spendserv1)
ex_s2 <- table(ex_spendserv2)

ex_j1 <- table(ex_jobs1)
ex_j2 <- table(ex_jobs2)

ex_a1 <- table(ex_aid1)
ex_a2 <- table(ex_aid2)

# descriptives 
ex_sprop1 <- round((ex_s1["1"]/sum(ex_s1))*100,1)
ex_sprop2 <- round((ex_s2["1"]/sum(ex_s2))*100,1)
ex_stest <- prop.test(c(ex_s1["1"],ex_s2["1"]),c(sum(ex_s1),sum(ex_s2)))

ex_jprop1 <- round((ex_j1["1"]/sum(ex_j1))*100,1)
ex_jprop2 <- round((ex_j2["1"]/sum(ex_j2))*100,1)
ex_jtest <- prop.test(c(ex_j1["1"],ex_j2["1"]),c(sum(ex_j1),sum(ex_j2)))

ex_aprop1 <- round((ex_a1["1"]/sum(ex_a1))*100,1)
ex_aprop2 <- round((ex_a2["1"]/sum(ex_a2))*100,1)
ex_atest <- prop.test(c(ex_a1["1"],ex_a2["1"]),c(sum(ex_a1),sum(ex_a2)))

# print
ex_tab <- 
rbind(	c(paste0(ex_sprop1,"%"),paste0(ex_sprop2,"%"),paste0(round(ex_stest$statistic,2),", p=",round(ex_stest$p.value,2))),
		c(paste0(ex_jprop1,"%"),paste0(ex_jprop2,"%"),paste0(round(ex_jtest$statistic,2),", p=",round(ex_jtest$p.value,2))),
		c(paste0(ex_aprop1,"%"),paste0(ex_aprop2,"%"),paste0(round(ex_atest$statistic,2),", p=",round(ex_atest$p.value,2))) )
rownames(ex_tab) <- c("Spending/Services", "Jobs and Standard of Living", "Aid to Blacks")
colnames(ex_tab) <- c("Standard Wording", "Stop-and-Think", "Difference (chi2)")

cat('\\caption{Extremity Under Standard and Stop-and-Think Wordings, ANES 1989 Pilot}\\label{tab:exzallerfeldman}', file='tables/zfextremity.txt')
xtable(ex_tab, align=c("l",rep("r",3)), file='tables/zfextremity.txt', append=TRUE)

